1 Summary
2 Initialization
3 Goodness-of-fit tests
- 3.1 Kullback–Leibler divergence
- 3.2 Jensen–Shannon divergence (also known as information radius and total divergence to the average)
4 General overview
5 Detailed goodness-of-fit test results
References

1 Summary

To examine the samplers’ uniformity, we ran them to generate samples corresponding to \(\alpha = 0.01, \beta = 0.01\), and \(w = 0.1\). The timeout for each sample generation was set to one hour. In total, 373.5 hours (15.56 days) of CPU time were needed for generating the samples (or reaching the timeout).

2 Initialization

2.1 Importing packages and installing them if needed

REQUIRED_PACKAGES <- 
  c("tidyverse", "gridExtra", "grid", "directlabels", "ggrepel","splines", 
    "modelr", "scales", "kableExtra")
lapply(
  REQUIRED_PACKAGES,
  function(pkg) {
    print(pkg)
    if (system.file(package = pkg) == "") {
      install.packages(pkg,
                       repos = "http://cran.us.r-project.org"
      )
    }
    do.call("library", list(pkg))
  }
)

## -- Attaching packages --------------------------------------------------- tidyverse 1.3.0 --

## v ggplot2 3.3.2     v purrr   0.3.4
## v tibble  3.0.1     v dplyr   0.8.5
## v tidyr   1.0.3     v stringr 1.4.0
## v readr   1.3.1     v forcats 0.5.0

## -- Conflicts ------------------------------------------------------ tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x dplyr::lag()    masks stats::lag()

## 
## Attaching package: 'gridExtra'

## The following object is masked from 'package:dplyr':
## 
##     combine

## 
## Attaching package: 'scales'

## The following object is masked from 'package:purrr':
## 
##     discard

## The following object is masked from 'package:readr':
## 
##     col_factor

## 
## Attaching package: 'kableExtra'

## The following object is masked from 'package:dplyr':
## 
##     group_rows

sessionInfo()

## R version 4.0.0 (2020-04-24)
## Platform: x86_64-w64-mingw32/x64 (64-bit)
## Running under: Windows 10 x64 (build 19042)
## 
## Matrix products: default
## 
## locale:
## [1] LC_COLLATE=Spanish_Spain.1252  LC_CTYPE=Spanish_Spain.1252   
## [3] LC_MONETARY=Spanish_Spain.1252 LC_NUMERIC=C                  
## [5] LC_TIME=Spanish_Spain.1252    
## 
## attached base packages:
## [1] splines   grid      stats     graphics  grDevices utils     datasets 
## [8] methods   base     
## 
## other attached packages:
##  [1] kableExtra_1.3.1       scales_1.1.1           modelr_0.1.8          
##  [4] ggrepel_0.9.1          directlabels_2021.1.13 gridExtra_2.3         
##  [7] forcats_0.5.0          stringr_1.4.0          dplyr_0.8.5           
## [10] purrr_0.3.4            readr_1.3.1            tidyr_1.0.3           
## [13] tibble_3.0.1           ggplot2_3.3.2          tidyverse_1.3.0       
## 
## loaded via a namespace (and not attached):
##  [1] tidyselect_1.1.0  xfun_0.19         haven_2.2.0       lattice_0.20-41  
##  [5] colorspace_1.4-1  vctrs_0.3.0       generics_0.0.2    viridisLite_0.3.0
##  [9] htmltools_0.4.0   yaml_2.2.1        rlang_0.4.6       pillar_1.4.4     
## [13] glue_1.4.1        withr_2.3.0       DBI_1.1.0         dbplyr_1.4.3     
## [17] readxl_1.3.1      lifecycle_0.2.0   munsell_0.5.0     gtable_0.3.0     
## [21] cellranger_1.1.0  rvest_0.3.5       evaluate_0.14     knitr_1.28       
## [25] fansi_0.4.1       broom_0.5.6       Rcpp_1.0.4.6      backports_1.1.6  
## [29] webshot_0.5.2     jsonlite_1.7.2    fs_1.4.1          hms_0.5.3        
## [33] digest_0.6.25     stringi_1.4.6     quadprog_1.5-8    cli_2.0.2        
## [37] tools_4.0.0       magrittr_1.5      crayon_1.3.4      pkgconfig_2.0.3  
## [41] ellipsis_0.3.1    xml2_1.3.2        reprex_0.3.0      lubridate_1.7.8  
## [45] assertthat_0.2.1  rmarkdown_2.5     httr_1.4.2        rstudioapi_0.11  
## [49] R6_2.4.1          nlme_3.1-147      compiler_4.0.0

2.2 Model constants and variables initialization

MODELS_PATH <- "../data"
MODELS_EXTENSIONS <-
  c("bdd", "kus", "quicksampler", "smarch", "spur", "unigen2")
models <- list.dirs(path = MODELS_PATH, 
                    full.names = FALSE, 
                    recursive = FALSE)
SAMPLER_COLORS <- c("#E91E63", "#7EBD5E", "#3388CC", "#FA9D0D", "#666666", "#FF6E66")
SPL_MODELS <- c("axtls", "busybox", "DellSPLOT", "embtoolkit-onlybool", "fiasco", 
                "jhipster", "LargeAutomotive", "toybox", "uClibc")
FORMATTED_SPL_MODELS <- c("axTLS", "BusyBox", "DellSPLOT", "EmbToolkit", "Fiasco", 
                          "JHipster", "LargeAutomotive", "ToyBox", "uClibc")
SAMPLERS <- c("BDDSampler", "KUS", "QuickSampler", "Smarch", "Spur", "Unigen2")

2.3 Auxiliary functions and variables for plots

pretty_name <- function(id) {
  if (id == "bdd") {
    "BDDSampler"  
  } else if (id == "quicksampler") {
    "Quicksampler"
  } else if (id == "smarch") {
    "Smarch"
  } else if (id == "spur") {
    "Spur"
  } else if (id == "unigen2") {
    "Unigen2"
  } else if (id == "kus") {
    "KUS"
  } else {
    stop("error in pretty_name")
  }
}

g_legend <- function(a.gplot){ 
  tmp <- ggplot_gtable(ggplot_build(a.gplot)) 
  leg <- which(sapply(tmp$grobs, function(x) x$name) == "guide-box") 
  legend <- tmp$grobs[[leg]] 
  legend
} 

blank_plot <-
  theme(axis.title.x=element_blank(),
        axis.text.x=element_blank(),
        axis.ticks.x=element_blank(),
        axis.title.y=element_blank(),
        axis.text.y=element_blank(),
        axis.ticks.y=element_blank()
  )

2.4 Analysis result tibble initialization

analysis_results <- 
  tibble(model = character(), 
         bdd_jsd = numeric(), 
         bdd_p_value = numeric(), 
         kus_jsd = numeric(), 
         kus_p_value = numeric(),
         quicksampler_jsd = numeric(), 
         quicksampler_p_value = numeric(), 
         smarch_jsd = numeric(), 
         smarch_p_value = numeric(), 
         spur_jsd = numeric(), 
         spur_p_value = numeric(), 
         unigen2_jsd = numeric(), 
         unigen2_p_value = numeric()
  )

3 Goodness-of-fit tests

3.1 Kullback–Leibler divergence

For discrete probability distributions \(P\) and \(Q\) defined on the same probability space \(\mathcal{X}\), the Kullback–Leibler divergence [1] from \(Q\) to \(P\) is defined as:

\[D_\mathrm{KL}(P||Q)=\sum_{x \in \mathcal{X}} P(x)\mathrm{log}_2\Big{(}\frac{P(x)}{Q(x)}\Big{)}\]

For the extreme cases, where \(P(x)\) or \(Q(x)\) are equal to zero, the directions given by Drost(2018) are followed to compute a numercially stable Kullback–Leibler divergence.

kullback_leibler_divergence <- function(p, q) {
  Q_adjustment <- 0.00001
  kld <- function(x, y) {
    if (x == 0) {
      0
    } else if (y == 0) {
      x*log(x/Q_adjustment, base=2)
    } else {
      x*log(x/y, base=2)
    }
  }
  sum(map2_dbl(p, q, kld))
}

3.2 Jensen–Shannon divergence (also known as information radius and total divergence to the average)

The Jensen–Shannon divergence (JSD) is a symmetrized and smoothed version of the Kullback–Leibler divergence [1] defined as: \[\mathrm{JSD(P||Q)}=\frac{1}{2}D_\mathrm{KL}(P||M)+\frac{1}{2}D_\mathrm{KL}(Q||M)\] where \(M=\frac{1}{2}(P+Q)\)

jensen_shannon_divergence <- function(p, q) {
  m <- (p+q)/2
  jsd <- kullback_leibler_divergence(p,m)/2 + kullback_leibler_divergence(q,m)/2
  ifelse(jsd<1, jsd, 1)
}

Let \(s\) be the sample size, and \(m\) the number of elements in \(P\) that are neither zero nor one (e.g., the JHipster model has 45 features, there are seven core features and no dead features, so \(m=38\)). According to the proof given by Grosse et al. in Section 4.C of [2], \(2 s (\mathrm{ln} 2) D(F, P)\) has a \(\chi^2\) distribution with \(m-1\) degrees of freedom. As a result, a [3] built upon the statistic \(2 s (\mathrm{ln} 2) D(F, P)\) will help us to decide whether the sampler is uniform.

for (m in models) {
  
  writeLines(str_c("Goodness of fit test of ", m))

  # Create and initialize variables to store divergences and p-values
  for (ext in MODELS_EXTENSIONS) {
    eval(parse(text=str_c(ext, "_jsd <- NA")))
    eval(parse(text=str_c(ext, "_p_value <- NA")))
  }

  # Import the model theoretical distribution
  th_path <- str_c(MODELS_PATH,
                   "/",
                   m,
                   "/population_desc")
  theoretical_distribution <- read_delim(
    file = str_c(th_path, "/", m, ".satdist"),
    delim = " ",
    col_names = FALSE,
    col_types = cols(
      col_integer(),
      col_character()
    )
  )
  colnames(theoretical_distribution) <- c("feature_num", "absolute_freqs")

  # Compute the theoretical probabilities
  absolute_freqs <- gmp::as.bigz(theoretical_distribution$absolute_freqs)
  total_solutions <- sum(absolute_freqs)
  theoretical_distribution$probabilities <-
    as.numeric(absolute_freqs/total_solutions)
  total_solutions <- as.numeric(total_solutions)
  theoretical_distribution <- theoretical_distribution %>%
    select("feature_num", "probabilities")
  
  # Analysis of each sampler
  plot_index <- 0
  histograms <- list()
  for (ext in MODELS_EXTENSIONS) {
    plot_index <- plot_index + 1

    # Import sample
    file_name <- str_c(MODELS_PATH,
                       "/",
                       m,
                       "/std_samples/",
                       m, "_satdist.", ext)
    if (!file.exists(file_name)) {
      empty_plot <-
        ggplot()+
        ggtitle(pretty_name(ext)) +
        annotate(geom="text", x=3, y=30, label="<< time out >>",
                 color="#E91E63",
                 size=8) +
        blank_plot
      histograms[[plot_index]] <- empty_plot
      next()
    }    
    sample <- read_delim(
      file = file_name,
      delim = ";",
      col_names = TRUE,
      col_types = cols(
        col_integer()
      )
    )
    colnames(sample) <- "feature_num"

    # Compute empirical frequencies
    empirical_distribution <- sample %>%
      count(feature_num) %>%
      mutate(absolute_freqs = n) %>% 
      arrange(feature_num) 
    sample_size <- nrow(sample) 
    empirical_distribution$frequencies <-
      empirical_distribution$absolute_freqs/sample_size
    empirical_distribution <- empirical_distribution %>%
      select(feature_num, frequencies)

    # Join theoretical and empirical information
    distributions <- left_join(
      theoretical_distribution,
      empirical_distribution, 
      by="feature_num") 
    # Set NA's in the empirical distribution to zero
    distributions[is.na(distributions$frequencies),]$frequencies <- 0
    # Remove rows with no configurations
    distributions <- filter(distributions, probabilities>0) 
    # Get jensen shannon divergence
    jsd <- jensen_shannon_divergence(distributions$probabilities,
                                     distributions$frequencies)
    # Get goodness-of-fit p-value
    X2 <- 2*sample_size*log(2)*jsd
    degrees_of_freedom <- nrow(distributions)-1
    p_value <- 1-pchisq(X2, degrees_of_freedom)
    
    # Store jsd and p_values in their corresponding variables
    eval(parse(text=str_c(ext, "_jsd <- ", jsd)))
    eval(parse(text=str_c(ext, "_p_value <- ", p_value)))
    
    # Prepare distribution data for plotting
    histogram <- gather(distributions, 
                        probabilities, frequencies, 
                        value="pr", key="Distribution")
    levels <- c("probabilities", "frequencies")
    histogram$Distribution <- factor(histogram$Distribution, levels=levels)
    histogram$Distribution <- fct_recode(histogram$Distribution, 
                                 "Theoretical" = "probabilities",
                                 "Empirical" = "frequencies"
                                )
    # Generate the histogram
    histograms[[plot_index]] <- 
      ggplot(histogram, 
        aes(x=feature_num, y=pr, fill=Distribution, col=Distribution)) +
        scale_fill_manual(values=c("#3388CC", "#E91E63")) +
        scale_color_manual(values=c("#3388CC", "#E91E63")) +
        geom_col(alpha=0.5, size=0.1, position="dodge2") +
        scale_x_continuous("#True variables per\nSAT-solution")+#, 
        scale_y_continuous("Probability")+
        ggtitle(pretty_name(ext))

  } # for (ext in MODELS_EXTENSIONS)
  
  analysis_results <- analysis_results %>%
    add_row(model = m, 
            bdd_jsd = bdd_jsd, 
            bdd_p_value = bdd_p_value, 
            kus_jsd = kus_jsd, 
            kus_p_value = kus_p_value,
            quicksampler_jsd = quicksampler_jsd, 
            quicksampler_p_value = quicksampler_p_value, 
            smarch_jsd = smarch_jsd, 
            smarch_p_value = smarch_p_value, 
            spur_jsd = spur_jsd, 
            spur_p_value = spur_p_value, 
            unigen2_jsd = unigen2_jsd, 
            unigen2_p_value = unigen2_p_value
    )
  
  legend <- g_legend(histograms[[1]])
  for (i in 1:6) {
    histograms[[i]] <- histograms[[i]] + 
    theme(legend.position = "none")
  }
  histograms_plot <- arrangeGrob(
      arrangeGrob(textGrob(
                    m, 
                    gp = gpar(fontsize = 18, font=2)), 
                  legend,
                  nrow=1, ncol=2,
                  widths=c(2,1)), 
      arrangeGrob(
        histograms[[1]], histograms[[2]], histograms[[3]], 
        histograms[[4]], histograms[[5]], histograms[[6]],
        nrow=2),
      nrow=2,
      heights=c(1,6)
    )
  grid.arrange(histograms_plot)
  file_name <- str_c(MODELS_PATH,
                     "/",
                     m,
                     "/goodness_of_fit/",
                     m,
                     "_hist.pdf")
  ggsave(file_name, histograms_plot, width=15, height=6)
  grid.arrange(histograms_plot)
} # for (m in models)

## Goodness of fit test of 10.sk_1_46

## Goodness of fit test of 107.sk_3_90

## Goodness of fit test of 109.sk_4_36

## Goodness of fit test of 110.sk_3_88

## Goodness of fit test of 111.sk_2_36

## Goodness of fit test of 19.sk_3_48

## Goodness of fit test of 27.sk_3_32

## Goodness of fit test of 32.sk_4_38

## Goodness of fit test of 35.sk_3_52

## Goodness of fit test of 53.sk_4_32

## Goodness of fit test of 55.sk_3_46

## Goodness of fit test of 77.sk_3_44

## Goodness of fit test of 84.sk_4_77

## Goodness of fit test of axtls

## Goodness of fit test of blasted_case_0_b11_1

## Goodness of fit test of blasted_case_0_b12_1

## Goodness of fit test of blasted_case_0_b12_2

## Goodness of fit test of blasted_case_1_b11_1

## Goodness of fit test of blasted_case_1_b12_1

## Goodness of fit test of blasted_case_1_b12_2

## Goodness of fit test of blasted_case_1_b14_1

## Goodness of fit test of blasted_case_1_b14_2

## Goodness of fit test of blasted_case_1_b14_3

## Goodness of fit test of blasted_case_2_b12_1

## Goodness of fit test of blasted_case_2_b12_2

## Goodness of fit test of blasted_case_2_b14_1

## Goodness of fit test of blasted_case_2_b14_2

## Goodness of fit test of blasted_case_2_b14_3

## Goodness of fit test of blasted_case_3_b14_1

## Goodness of fit test of blasted_case_3_b14_2

## Goodness of fit test of blasted_case_3_b14_3

## Goodness of fit test of blasted_case1

## Goodness of fit test of blasted_case10

## Goodness of fit test of blasted_case100

## Goodness of fit test of blasted_case101

## Goodness of fit test of blasted_case102

## Goodness of fit test of blasted_case103

## Goodness of fit test of blasted_case105

## Goodness of fit test of blasted_case106

## Goodness of fit test of blasted_case108

## Goodness of fit test of blasted_case109

## Goodness of fit test of blasted_case11

## Goodness of fit test of blasted_case110

## Goodness of fit test of blasted_case111

## Goodness of fit test of blasted_case112

## Goodness of fit test of blasted_case113

## Goodness of fit test of blasted_case114

## Goodness of fit test of blasted_case115

## Goodness of fit test of blasted_case116

## Goodness of fit test of blasted_case117

## Goodness of fit test of blasted_case118

## Goodness of fit test of blasted_case119

## Goodness of fit test of blasted_case120

## Goodness of fit test of blasted_case121

## Goodness of fit test of blasted_case122

## Goodness of fit test of blasted_case123

## Goodness of fit test of blasted_case124

## Goodness of fit test of blasted_case125

## Goodness of fit test of blasted_case126

## Goodness of fit test of blasted_case127

## Goodness of fit test of blasted_case128

## Goodness of fit test of blasted_case130

## Goodness of fit test of blasted_case131

## Goodness of fit test of blasted_case132

## Goodness of fit test of blasted_case133

## Goodness of fit test of blasted_case134

## Goodness of fit test of blasted_case135

## Goodness of fit test of blasted_case136

## Goodness of fit test of blasted_case137

## Goodness of fit test of blasted_case14

## Goodness of fit test of blasted_case144

## Goodness of fit test of blasted_case145

## Goodness of fit test of blasted_case146

## Goodness of fit test of blasted_case15

## Goodness of fit test of blasted_case17

## Goodness of fit test of blasted_case19

## Goodness of fit test of blasted_case2

## Goodness of fit test of blasted_case20

## Goodness of fit test of blasted_case200

## Goodness of fit test of blasted_case201

## Goodness of fit test of blasted_case202

## Goodness of fit test of blasted_case203

## Goodness of fit test of blasted_case204

## Goodness of fit test of blasted_case205

## Goodness of fit test of blasted_case206

## Goodness of fit test of blasted_case207

## Goodness of fit test of blasted_case208

## Goodness of fit test of blasted_case21

## Goodness of fit test of blasted_case210

## Goodness of fit test of blasted_case211

## Goodness of fit test of blasted_case213

## Goodness of fit test of blasted_case214

## Goodness of fit test of blasted_case22

## Goodness of fit test of blasted_case23

## Goodness of fit test of blasted_case24

## Goodness of fit test of blasted_case25

## Goodness of fit test of blasted_case26

## Goodness of fit test of blasted_case27

## Goodness of fit test of blasted_case28

## Goodness of fit test of blasted_case29

## Goodness of fit test of blasted_case3

## Goodness of fit test of blasted_case30

## Goodness of fit test of blasted_case31

## Goodness of fit test of blasted_case32

## Goodness of fit test of blasted_case33

## Goodness of fit test of blasted_case34

## Goodness of fit test of blasted_case35

## Goodness of fit test of blasted_case36

## Goodness of fit test of blasted_case38

## Goodness of fit test of blasted_case39

## Goodness of fit test of blasted_case4

## Goodness of fit test of blasted_case40

## Goodness of fit test of blasted_case41

## Goodness of fit test of blasted_case43

## Goodness of fit test of blasted_case44

## Goodness of fit test of blasted_case45

## Goodness of fit test of blasted_case46

## Goodness of fit test of blasted_case47

## Goodness of fit test of blasted_case5

## Goodness of fit test of blasted_case50

## Goodness of fit test of blasted_case51

## Goodness of fit test of blasted_case52

## Goodness of fit test of blasted_case53

## Goodness of fit test of blasted_case54

## Goodness of fit test of blasted_case55

## Goodness of fit test of blasted_case56

## Goodness of fit test of blasted_case57

## Goodness of fit test of blasted_case58

## Goodness of fit test of blasted_case59

## Goodness of fit test of blasted_case59_1

## Goodness of fit test of blasted_case6

## Goodness of fit test of blasted_case60

## Goodness of fit test of blasted_case61

## Goodness of fit test of blasted_case62

## Goodness of fit test of blasted_case63

## Goodness of fit test of blasted_case64

## Goodness of fit test of blasted_case68

## Goodness of fit test of blasted_case7

## Goodness of fit test of blasted_case8

## Goodness of fit test of blasted_case9

## Goodness of fit test of blasted_squaring22

## Goodness of fit test of blasted_squaring26

## Goodness of fit test of blasted_squaring50

## Goodness of fit test of blasted_squaring51

## Goodness of fit test of busybox

## Goodness of fit test of DellSPLOT

## Goodness of fit test of embtoolkit-onlybool

## Goodness of fit test of fiasco

## Goodness of fit test of GuidanceService2.sk_2_27

## Goodness of fit test of jhipster

## Goodness of fit test of LargeAutomotive

## Goodness of fit test of polynomial.sk_7_25

## Goodness of fit test of registerlesSwap.sk_3_10

## Goodness of fit test of s1196a_3_2

## Goodness of fit test of s1196a_7_4

## Goodness of fit test of s1238a_3_2

## Goodness of fit test of s1238a_7_4

## Goodness of fit test of s1488_15_7

## Goodness of fit test of s1488_7_4

## Goodness of fit test of s27_15_7

## Goodness of fit test of s27_3_2

## Goodness of fit test of s27_7_4

## Goodness of fit test of s27_new_15_7

## Goodness of fit test of s27_new_3_2

## Goodness of fit test of s27_new_7_4

## Goodness of fit test of s298_15_7

## Goodness of fit test of s298_3_2

## Goodness of fit test of s298_7_4

## Goodness of fit test of s344_15_7

## Goodness of fit test of s344_3_2

## Goodness of fit test of s344_7_4

## Goodness of fit test of s349_15_7

## Goodness of fit test of s349_3_2

## Goodness of fit test of s349_7_4

## Goodness of fit test of s382_15_7

## Goodness of fit test of s382_3_2

## Goodness of fit test of s382_7_4

## Goodness of fit test of s420_15_7

## Goodness of fit test of s420_3_2

## Goodness of fit test of s420_7_4

## Goodness of fit test of s420_new_15_7

## Goodness of fit test of s420_new_3_2

## Goodness of fit test of s420_new_7_4

## Goodness of fit test of s420_new1_15_7

## Goodness of fit test of s420_new1_3_2

## Goodness of fit test of s420_new1_7_4

## Goodness of fit test of s444_15_7

## Goodness of fit test of s444_3_2

## Goodness of fit test of s444_7_4

## Goodness of fit test of s510_15_7

## Goodness of fit test of s510_3_2

## Goodness of fit test of s510_7_4

## Goodness of fit test of s526_15_7

## Goodness of fit test of s526_3_2

## Goodness of fit test of s526_7_4

## Goodness of fit test of s526a_15_7

## Goodness of fit test of s526a_3_2

## Goodness of fit test of s526a_7_4

## Goodness of fit test of s641_15_7

## Goodness of fit test of s641_3_2

## Goodness of fit test of s641_7_4

## Goodness of fit test of s713_15_7

## Goodness of fit test of s713_3_2

## Goodness of fit test of s713_7_4

## Goodness of fit test of s820a_15_7

## Goodness of fit test of s820a_3_2

## Goodness of fit test of s820a_7_4

## Goodness of fit test of s832a_15_7

## Goodness of fit test of s832a_3_2

## Goodness of fit test of s832a_7_4

## Goodness of fit test of s838_15_7

## Goodness of fit test of s838_3_2

## Goodness of fit test of s838_7_4

## Goodness of fit test of s953a_3_2

## Goodness of fit test of s953a_7_4

## Goodness of fit test of tableBasedAddition.sk_240_1024

## Goodness of fit test of toybox

## Goodness of fit test of uClibc

4 General overview

overview <- analysis_results[,seq(1,13,by=2)] 
colnames(overview)[2:7] <- SAMPLERS
rejected <- rep(0, 6)
for(i in 2:7) {
  non_nan_p_values <- overview[[i]][!is.na(overview[[i]])]
  rejected[i-1] <- 100*sum(non_nan_p_values<0.01)/length(non_nan_p_values)
}
rejected_summary <- tibble(
  sampler = SAMPLERS,
  rejected = str_c(round(rejected, 2), "% rejected")
)
for (i in 2:7) {
  overview[[i]] <- cut(overview[[i]], breaks=seq(0, 1, by=0.1), include.lowest=TRUE, na.rm=TRUE)
}
overview <- overview %>% 
  gather(2:7, key="sampler", value="p_value") %>%
  filter(!is.na(p_value))
models_per_sampler <- overview %>% 
  group_by(sampler) %>%
  summarize(total=n())
p_values_per_sampler <- overview %>%
    group_by(sampler, p_value) %>%
    summarize(p_value_count=n())
summary_table <- 
  inner_join(models_per_sampler, p_values_per_sampler) %>%
  mutate(percentage = 100*p_value_count/total)

## Joining, by = "sampler"

levels <- c("[0,0.1]", "(0.1,0.2]", "(0.2,0.3]", "(0.3,0.4]", "(0.5,0.6]", "(0.6,0.7]",
            "(0.7,0.8]", "(0.8,0.9]", "(0.9,1]")
summary_table$p_value <- factor(summary_table$p_value, levels=levels)
ggplot(summary_table, aes(x=p_value, y=percentage, fill=sampler)) +
  scale_fill_manual(values=SAMPLER_COLORS) +
  geom_bar(stat="identity", color="black", size=0.1) +
  facet_wrap(.~sampler) +
  geom_text(x = 5, y = 70, aes(label = rejected), data = rejected_summary, size=3) +
  theme_bw() +
  theme(legend.position = "none") +
  theme(axis.text.x = element_text (angle=60, hjust=1)) +
  scale_x_discrete("p-value") +
  scale_y_continuous("%Samples")

ggsave("test_results.pdf", width = 4.8, height = 3.5)

5 Detailed goodness-of-fit test results

In the following table:

bdd_jsd and bdd_p_value stand for the Jensen-Shanon divergences and p-values obtained with BDDSampler, respectively.
kus_jsd and kus_p_value stand for the Jensen-Shanon divergences and p-values obtained with KUS, respectively.
…
unigen2_jsd and unigen2_p_value stand for the Jensen-Shanon divergences and p-values obtained with Unigen2, respectively.

write.table(
  analysis_results,
  file = str_c(MODELS_PATH,"/goodness_of_fit.csv"),
  sep = ";",
  row.names = FALSE
)
knitr::kable(analysis_results)

model	bdd_jsd	bdd_p_value	kus_jsd	kus_p_value	quicksampler_jsd	quicksampler_p_value	smarch_jsd	smarch_p_value	spur_jsd	spur_p_value	unigen2_jsd	unigen2_p_value
10.sk_1_46	0.0013316	1.0000000	0.0010786	1.0000000	0.1303615	0.0000000	NA	NA	0.0011061	1.0000000	NA	NA
107.sk_3_90	0.0011242	1.0000000	0.0011822	1.0000000	0.5392516	0.0000000	NA	NA	0.0011434	1.0000000	NA	NA
109.sk_4_36	0.0011793	1.0000000	0.0013787	1.0000000	0.4529571	0.0000000	NA	NA	0.0017584	1.0000000	NA	NA
110.sk_3_88	0.0016989	1.0000000	NA	NA	0.4763476	0.0000000	NA	NA	0.0020305	1.0000000	NA	NA
111.sk_2_36	0.0012871	0.9999995	0.0010419	1.0000000	0.6147990	0.0000000	NA	NA	0.0010214	1.0000000	NA	NA
19.sk_3_48	0.0014507	1.0000000	0.0014589	1.0000000	0.2438726	0.0000000	NA	NA	0.0014534	1.0000000	NA	NA
27.sk_3_32	0.0008328	0.9999999	0.0014774	0.9998948	0.2736768	0.0000000	NA	NA	0.0012461	0.9999872	NA	NA
32.sk_4_38	0.0014524	0.9999993	0.0011134	1.0000000	0.1098637	0.0000000	NA	NA	0.0010514	1.0000000	NA	NA
35.sk_3_52	0.0011267	1.0000000	0.0010216	1.0000000	0.0758962	0.0000000	NA	NA	0.0009918	1.0000000	NA	NA
53.sk_4_32	0.0016238	0.9999840	0.0007717	1.0000000	0.0703430	0.0000000	NA	NA	0.0016278	0.9999834	NA	NA
55.sk_3_46	0.0009962	1.0000000	0.0016969	1.0000000	0.2433804	0.0000000	NA	NA	0.0011450	1.0000000	NA	NA
77.sk_3_44	0.0015424	0.9999999	0.0010112	1.0000000	0.2816439	0.0000000	NA	NA	0.0013167	1.0000000	NA	NA
84.sk_4_77	0.0008614	1.0000000	0.0014515	1.0000000	0.4004079	0.0000000	NA	NA	0.0012513	1.0000000	NA	NA
axtls	0.0007275	1.0000000	0.0011498	1.0000000	0.3379597	0.0000000	NA	NA	0.0006899	1.0000000	NA	NA
blasted_case_0_b11_1	0.0019278	1.0000000	0.0417133	0.0000000	0.0997540	0.0000000	NA	NA	0.0019610	1.0000000	NA	NA
blasted_case_0_b12_1	0.0029611	1.0000000	0.4688336	0.0000000	0.2526064	0.0000000	NA	NA	0.0028220	1.0000000	NA	NA
blasted_case_0_b12_2	0.0033852	1.0000000	0.0788632	0.0000000	0.0232519	0.0000013	NA	NA	0.0033285	1.0000000	NA	NA
blasted_case_1_b11_1	0.0022914	1.0000000	0.1195219	0.0000000	0.0739621	0.0000000	NA	NA	0.0023048	1.0000000	NA	NA
blasted_case_1_b12_1	0.0024868	1.0000000	0.3545621	0.0000000	0.0240123	0.0000001	NA	NA	0.0025873	1.0000000	NA	NA
blasted_case_1_b12_2	0.0032931	1.0000000	0.2577323	0.0000000	0.0462589	0.0000000	NA	NA	0.0030283	1.0000000	NA	NA
blasted_case_1_b14_1	0.0018851	1.0000000	0.0017869	1.0000000	0.4632492	0.0000000	NA	NA	0.0016259	1.0000000	0.0110824	1.0000000
blasted_case_1_b14_2	0.0014742	1.0000000	0.0016105	1.0000000	0.4078432	0.0000000	NA	NA	0.0018392	1.0000000	0.0113253	1.0000000
blasted_case_1_b14_3	0.0016818	1.0000000	0.0015429	1.0000000	0.5278118	0.0000000	NA	NA	0.0021431	1.0000000	NA	NA
blasted_case_2_b12_1	0.0028778	1.0000000	0.3232143	0.0000000	0.0765531	0.0000000	NA	NA	0.0022554	1.0000000	NA	NA
blasted_case_2_b12_2	0.0036842	1.0000000	0.1826947	0.0000000	0.0260535	0.0000008	NA	NA	0.0035877	1.0000000	NA	NA
blasted_case_2_b14_1	0.0018851	1.0000000	0.0012670	1.0000000	0.3929843	0.0000000	NA	NA	0.0016464	1.0000000	0.0149023	1.0000000
blasted_case_2_b14_2	0.0018100	1.0000000	0.0013717	1.0000000	0.3597543	0.0000000	NA	NA	0.0013334	1.0000000	0.0166888	1.0000000
blasted_case_2_b14_3	0.0020891	1.0000000	0.0018365	1.0000000	0.1725337	0.0000000	NA	NA	0.0022225	1.0000000	NA	NA
blasted_case_3_b14_1	0.0016591	1.0000000	0.0018600	1.0000000	0.4719370	0.0000000	NA	NA	0.0012666	1.0000000	0.0118623	1.0000000
blasted_case_3_b14_2	0.0019442	1.0000000	0.0023708	1.0000000	0.3347211	0.0000000	NA	NA	0.0017235	1.0000000	0.0139701	1.0000000
blasted_case_3_b14_3	0.0014045	1.0000000	0.0018063	1.0000000	0.5344324	0.0000000	NA	NA	0.0013614	1.0000000	NA	NA
blasted_case1	0.0016826	1.0000000	0.0009507	1.0000000	0.1988378	0.8339750	NA	NA	0.0017658	0.9999999	0.0109720	1.0000000
blasted_case10	0.0007979	1.0000000	0.0011938	1.0000000	0.3494668	0.0000000	NA	NA	0.0012424	1.0000000	NA	NA
blasted_case100	0.0010896	1.0000000	0.0007843	1.0000000	0.0925407	0.2607446	0.0100337	1.0000000	0.0012345	1.0000000	0.0075227	1.0000000
blasted_case101	0.0014500	1.0000000	0.0015283	1.0000000	0.1954709	0.0000000	0.0136956	1.0000000	0.0012776	1.0000000	0.0131078	1.0000000
blasted_case102	0.0003854	0.9995032	0.0003133	0.9998448	0.0231344	0.9090618	0.0000286	1.0000000	0.0007476	0.9851516	0.0037826	0.9997615
blasted_case103	0.0008751	0.9698405	0.0003510	0.9997050	0.0026038	0.9999994	0.0000286	1.0000000	0.0005778	0.9957505	0.0039667	0.9996884
blasted_case105	0.0012287	1.0000000	0.0014819	1.0000000	0.6248556	0.0000000	NA	NA	0.0008722	1.0000000	0.0109131	1.0000000
blasted_case106	0.0018708	1.0000000	0.0008649	1.0000000	0.5739558	0.0000000	NA	NA	0.0014795	1.0000000	0.0159217	1.0000000
blasted_case108	0.0023707	1.0000000	0.0022013	1.0000000	0.0678261	0.0000000	NA	NA	0.0023988	1.0000000	0.0205054	1.0000000
blasted_case109	0.0023712	1.0000000	0.0021783	1.0000000	0.1485599	0.0000000	NA	NA	0.0029915	1.0000000	0.0173750	1.0000000
blasted_case11	0.0008591	0.9999999	0.0010693	0.9999983	0.0453019	0.9475705	NA	NA	0.0012212	0.9999901	0.0126112	0.9999667
blasted_case110	0.0014911	0.9999998	0.0011653	1.0000000	0.0081234	0.0593937	NA	NA	0.0012856	1.0000000	0.0081647	1.0000000
blasted_case111	0.0015777	0.9999996	0.0013906	1.0000000	0.0081556	0.0349344	NA	NA	0.0018419	0.9999949	NA	NA
blasted_case112	0.0008276	1.0000000	0.2058916	0.0000000	0.0277496	0.0000000	NA	NA	0.0009899	0.9999998	0.0090163	0.9999994
blasted_case113	0.0016337	0.9999997	0.0011378	1.0000000	0.0088883	0.0033368	NA	NA	0.0014250	1.0000000	NA	NA
blasted_case114	0.0023924	1.0000000	0.0016931	1.0000000	0.3825535	0.0000000	NA	NA	0.0021159	1.0000000	NA	NA
blasted_case115	0.0020164	1.0000000	0.0019272	1.0000000	0.7008073	0.0000000	NA	NA	0.0016360	1.0000000	NA	NA
blasted_case116	0.0020569	1.0000000	0.4346648	0.0000000	0.1682556	0.0000000	NA	NA	0.0018352	1.0000000	NA	NA
blasted_case117	0.0012065	1.0000000	0.0014203	1.0000000	0.0171544	0.0000000	NA	NA	0.0011701	1.0000000	NA	NA
blasted_case118	0.0012810	1.0000000	0.0012636	1.0000000	0.0049857	0.6586274	NA	NA	0.0011136	1.0000000	NA	NA
blasted_case119	0.0018581	1.0000000	0.0797089	0.0000000	0.4994645	0.0000000	NA	NA	0.0019467	1.0000000	0.0205294	1.0000000
blasted_case120	0.0020628	1.0000000	0.0584869	0.0000000	0.2793107	0.0000000	NA	NA	0.0018295	1.0000000	NA	NA
blasted_case121	0.0017096	1.0000000	0.0019320	1.0000000	0.0465179	0.0000000	NA	NA	0.0015757	1.0000000	0.0125992	1.0000000
blasted_case122	0.0011936	1.0000000	0.0009375	1.0000000	0.2876590	0.0000000	NA	NA	0.0016608	0.9999977	NA	NA
blasted_case123	0.0013649	1.0000000	0.0266899	0.0000000	0.3467390	0.0000000	NA	NA	0.0013202	1.0000000	0.0080562	1.0000000
blasted_case124	0.0015303	1.0000000	0.0017748	1.0000000	0.4583823	0.0000000	NA	NA	0.0013057	1.0000000	0.0138454	1.0000000
blasted_case125	0.0014193	1.0000000	0.0012320	1.0000000	0.3326160	0.0000000	NA	NA	0.0018344	1.0000000	NA	NA
blasted_case126	0.0017141	1.0000000	0.0013343	1.0000000	0.2419164	0.0000000	NA	NA	0.0009272	1.0000000	NA	NA
blasted_case127	0.0005416	0.9445866	0.0001098	0.9997753	0.0004725	1.0000000	0.0004725	1.0000000	0.0004186	0.9747380	0.0005179	0.9999831
blasted_case128	0.0005416	0.9445866	0.0003419	0.9868785	0.0002259	1.0000000	0.0004725	1.0000000	0.0003239	0.9890396	0.0020782	0.9969058
blasted_case130	0.0018911	1.0000000	0.2137364	0.0000000	0.2872276	0.0000000	NA	NA	0.0017778	1.0000000	NA	NA
blasted_case131	0.0017268	1.0000000	0.0975177	0.0000000	0.4296307	0.0000000	NA	NA	0.0011836	1.0000000	NA	NA
blasted_case132	0.0015379	1.0000000	0.0014369	1.0000000	0.3391403	0.0000000	NA	NA	0.0013685	1.0000000	0.0114168	1.0000000
blasted_case133	0.0014676	1.0000000	0.0017505	1.0000000	0.4996645	0.0000000	NA	NA	0.0017825	1.0000000	NA	NA
blasted_case134	0.0012031	1.0000000	0.0010800	1.0000000	0.2331239	0.0000000	0.0127711	1.0000000	0.0012302	1.0000000	0.0125423	1.0000000
blasted_case135	0.0014394	1.0000000	0.0012609	1.0000000	0.3344309	0.0000000	NA	NA	0.0018227	1.0000000	0.0135920	1.0000000
blasted_case136	0.0017052	1.0000000	0.0018395	1.0000000	0.4187843	0.0000000	NA	NA	0.0012336	1.0000000	NA	NA
blasted_case137	0.0017052	0.9999986	0.0009536	1.0000000	0.0294039	0.0000048	0.0096525	1.0000000	0.0011012	1.0000000	0.0120360	1.0000000
blasted_case14	0.0011852	1.0000000	0.0094691	0.0589483	0.0567743	0.0000000	NA	NA	0.0016727	1.0000000	NA	NA
blasted_case144	0.0027271	1.0000000	0.0025736	1.0000000	0.4061649	0.0000000	NA	NA	0.0019940	1.0000000	NA	NA
blasted_case145	0.0011384	1.0000000	0.0011936	1.0000000	0.2452657	0.0000000	NA	NA	0.0011247	1.0000000	NA	NA
blasted_case146	0.0015591	1.0000000	0.0012580	1.0000000	0.1752916	0.0000000	NA	NA	0.0009134	1.0000000	NA	NA
blasted_case15	0.0011690	1.0000000	0.0015610	1.0000000	0.1917139	0.0000000	NA	NA	0.0013028	1.0000000	NA	NA
blasted_case17	0.0014954	0.9998125	0.0008796	0.9999998	0.0113537	0.9999996	0.0116591	0.9999962	0.0017960	0.9986162	0.0128243	0.9999850
blasted_case19	0.0024373	1.0000000	0.0385241	0.0000000	0.1428813	0.0000000	NA	NA	0.0019256	1.0000000	NA	NA
blasted_case2	0.0020627	1.0000000	0.0014367	1.0000000	0.1579749	0.0000000	NA	NA	0.0018493	1.0000000	0.0123748	1.0000000
blasted_case20	0.0020343	1.0000000	0.0129630	0.2644008	0.1542216	0.0000000	NA	NA	0.0028414	1.0000000	NA	NA
blasted_case200	0.0001413	0.9000586	0.0004033	0.6441214	0.0000000	1.0000000	0.1379254	0.9024450	0.0000902	0.9457864	NA	NA
blasted_case201	0.0014536	1.0000000	0.0011061	1.0000000	0.1952627	0.0000000	NA	NA	0.0017182	1.0000000	0.0121129	1.0000000
blasted_case202	0.0014536	1.0000000	0.0009882	1.0000000	0.3206113	0.0000000	NA	NA	0.0012026	1.0000000	0.0081080	1.0000000
blasted_case203	0.0011079	1.0000000	0.0012247	1.0000000	0.3229443	0.0000000	NA	NA	0.0011089	1.0000000	0.0090918	1.0000000
blasted_case204	0.0018557	1.0000000	0.0011121	1.0000000	0.3094139	0.0000000	NA	NA	0.0012490	1.0000000	0.0095621	1.0000000
blasted_case205	0.0011079	1.0000000	0.0012486	1.0000000	0.3724481	0.0000000	NA	NA	0.0011552	1.0000000	0.0098149	1.0000000
blasted_case206	0.0001421	0.8992123	0.0001195	0.9201797	0.0000000	1.0000000	0.1379254	0.9024450	0.0001959	0.8470039	NA	NA
blasted_case207	0.0013431	1.0000000	0.0010460	1.0000000	0.5121875	0.0000000	NA	NA	0.0012930	1.0000000	NA	NA
blasted_case208	0.0012305	1.0000000	0.0009939	1.0000000	0.5010501	0.0000000	NA	NA	0.0011204	1.0000000	NA	NA
blasted_case21	0.0010137	0.9999972	0.0010786	0.9999939	0.0358161	0.9813413	NA	NA	0.0012047	0.9999763	0.0111999	0.9999781
blasted_case210	0.0020923	1.0000000	0.0022516	1.0000000	0.5861683	0.0000000	NA	NA	0.0021938	1.0000000	NA	NA
blasted_case211	0.0021476	1.0000000	0.0020937	1.0000000	0.6154459	0.0000000	NA	NA	0.0025250	1.0000000	NA	NA
blasted_case213	0.0013722	1.0000000	0.0012998	1.0000000	0.2991038	0.0000000	NA	NA	0.0014452	1.0000000	NA	NA
blasted_case214	0.0012945	1.0000000	0.0014664	1.0000000	0.5036780	0.0000000	NA	NA	0.0015989	1.0000000	NA	NA
blasted_case22	0.0012935	0.9999447	0.0006666	1.0000000	0.1869638	0.0989761	NA	NA	0.0013351	0.9999200	0.0103280	0.9999919
blasted_case23	0.0011576	0.9999915	0.0012776	0.9999709	0.0395712	0.9990099	0.0087868	0.9999999	0.0009228	0.9999996	0.0095131	0.9999997
blasted_case24	0.0004206	0.9999989	0.0006107	0.9999688	0.0125787	0.9999985	0.0000716	1.0000000	0.0005936	0.9999756	0.0114299	0.9942758
blasted_case25	0.0009693	0.9912006	0.0004414	0.9999515	0.1382340	0.2980760	0.0000069	1.0000000	0.0006286	0.9994241	0.0089516	0.9886393
blasted_case26	0.0002250	0.9999973	0.0008145	0.9914855	0.0170156	0.8769486	0.0062134	0.9977760	0.0007287	0.9953640	0.0021523	0.9999972
blasted_case27	0.0008119	0.9916353	0.0004896	0.9995547	0.0432248	0.5604750	0.0008315	1.0000000	0.0007625	0.9940458	0.0067173	0.9960794
blasted_case28	0.0005210	0.9993479	0.0009358	0.9824850	0.0081984	0.9985542	0.0017588	0.9999994	0.0003331	0.9999622	0.0077963	0.9911335
blasted_case29	0.0007240	0.9998686	0.0007213	0.9998726	0.0102261	0.9999999	0.0000716	1.0000000	0.0006946	0.9999070	0.0127154	0.9882940
blasted_case3	0.0022203	1.0000000	0.0017513	1.0000000	0.3053524	0.0000000	NA	NA	0.0018026	1.0000000	0.0177045	1.0000000
blasted_case30	0.0004174	0.9999677	0.0004714	0.9999221	0.0863573	0.3691331	0.0000069	1.0000000	0.0004367	0.9999551	0.0035992	0.9999729
blasted_case31	0.0008331	0.9904030	0.0003728	0.9999210	0.0085349	0.9989658	0.0054084	0.9990193	0.0005700	0.9988783	0.0082956	0.9877055
blasted_case32	0.0007719	0.9936328	0.0005398	0.9991911	0.0039624	0.9999843	0.0029465	0.9999796	0.0004580	0.9997063	0.0037487	0.9998859
blasted_case33	0.0005342	0.9992410	0.0005246	0.9993203	0.1039727	0.0005682	0.0053979	0.9990307	0.0011252	0.9574154	0.0042879	0.9997317
blasted_case34	0.0023581	1.0000000	0.0048159	0.9903900	0.4083584	0.0000000	NA	NA	0.0013987	1.0000000	NA	NA
blasted_case35	0.0019160	1.0000000	0.0011863	1.0000000	0.1477567	0.0000000	NA	NA	0.0020315	1.0000000	NA	NA
blasted_case36	0.0006208	0.9939175	0.2326812	0.0000000	0.0051919	0.9999619	0.0018140	0.9999996	0.0005916	0.9952163	0.0082272	0.9540883
blasted_case38	0.0008133	0.9999977	0.0011353	0.9999046	0.0689595	0.9915912	0.0000006	1.0000000	0.0006154	0.9999999	0.0105105	0.9994004
blasted_case39	0.0007194	1.0000000	0.1606357	0.0000000	0.2939446	0.0000000	NA	NA	0.0011126	1.0000000	NA	NA
blasted_case4	0.0012104	0.9999997	0.0012092	0.9999997	0.0947795	0.5528688	NA	NA	0.0022558	0.9983168	0.0113389	0.9999999
blasted_case40	0.0010076	1.0000000	0.1644924	0.0000000	0.2403773	0.0000000	NA	NA	0.0011101	1.0000000	NA	NA
blasted_case41	0.0007194	1.0000000	0.1660845	0.0000000	0.2335599	0.0000000	NA	NA	0.0012302	1.0000000	NA	NA
blasted_case43	0.0011632	0.9999947	0.0007628	1.0000000	0.1568203	0.0080741	NA	NA	0.0011212	0.9999968	0.0096724	0.9999979
blasted_case44	0.0010830	0.9999998	0.0010262	0.9999999	0.2041506	0.2177300	NA	NA	0.0013682	0.9999942	0.0082610	0.9999998
blasted_case45	0.0007650	1.0000000	0.0012163	0.9999906	0.1112878	0.5943415	NA	NA	0.0011523	0.9999954	0.0075755	0.9999999
blasted_case46	0.0013910	0.9999956	0.0013299	0.9999977	0.2104658	0.8968999	NA	NA	0.0007067	1.0000000	0.0088611	0.9999998
blasted_case47	0.0012693	0.9998826	0.0012506	0.9999002	0.7102742	0.0000000	NA	NA	0.0004985	1.0000000	0.0065104	0.9999997
blasted_case5	0.0008064	1.0000000	0.0013215	1.0000000	0.0456971	0.0000000	NA	NA	0.0009391	1.0000000	0.0063748	1.0000000
blasted_case50	0.0024623	1.0000000	0.0021160	1.0000000	0.3571804	0.0000000	NA	NA	0.0024799	1.0000000	NA	NA
blasted_case51	0.0008130	0.9999998	0.0010555	0.9999953	0.4176279	0.0000000	NA	NA	0.0011509	0.9999864	0.0077257	0.9999989
blasted_case52	0.0013778	0.9998852	0.0011439	0.9999874	0.2779868	0.0000000	NA	NA	0.0011367	0.9999883	0.0065585	0.9999999
blasted_case53	0.0013778	0.9998852	0.0007551	0.9999999	0.0865416	0.7954681	NA	NA	0.0007449	1.0000000	0.0074541	0.9999993
blasted_case54	0.0016609	1.0000000	0.0017487	1.0000000	0.2636373	0.0000000	NA	NA	0.0016475	1.0000000	0.0139244	1.0000000
blasted_case55	0.0014842	0.9999998	0.0016966	0.9999979	0.4148133	0.9282980	NA	NA	0.0011513	1.0000000	0.0083362	1.0000000
blasted_case56	0.0017046	1.0000000	0.0016037	1.0000000	0.0408963	0.0000000	NA	NA	0.0012138	1.0000000	0.0134205	1.0000000
blasted_case57	0.0013478	1.0000000	0.0510739	0.0000000	0.1266625	0.0000000	NA	NA	0.0008728	1.0000000	0.0097858	1.0000000
blasted_case58	0.0010272	0.9999432	0.0006516	0.9999996	0.0181738	0.9787982	0.0067619	0.9999985	0.0006744	0.9999995	0.0040251	1.0000000
blasted_case59	0.0010423	0.9998872	0.1516599	0.0000000	0.1345081	0.0082167	0.0112438	0.9994616	0.0009698	0.9999455	0.0102664	0.9997502
blasted_case59_1	0.0013802	0.9984114	0.1459362	0.0000000	0.0216792	0.9673182	0.0078154	0.9999858	0.0008659	0.9999831	0.0054736	0.9999997
blasted_case6	0.0019353	1.0000000	0.0016612	1.0000000	0.3291754	0.0000000	NA	NA	0.0021152	1.0000000	NA	NA
blasted_case60	0.0002232	0.9595804	0.0002519	0.9477296	0.0000000	1.0000000	0.0319773	0.9848368	0.0004032	0.8665128	NA	NA
blasted_case61	0.0014460	1.0000000	0.0134048	0.0000572	0.2103115	0.0000000	NA	NA	0.0011016	1.0000000	NA	NA
blasted_case62	0.0016002	1.0000000	0.0013240	1.0000000	0.1184163	0.0000000	NA	NA	0.0020601	0.9999981	0.0082011	1.0000000
blasted_case63	0.0008590	0.9999917	0.2602539	0.0000000	0.0279455	0.9351566	0.0061343	0.9999995	0.0008255	0.9999946	0.0065191	0.9999988
blasted_case64	0.0005776	0.9999996	0.0006396	0.9999987	0.0136229	0.0000000	NA	NA	0.0008974	0.9999556	0.0107610	0.9992996
blasted_case68	0.0007683	1.0000000	0.0240921	0.0000000	0.1492359	0.1379597	NA	NA	0.0013657	1.0000000	0.0124779	1.0000000
blasted_case7	0.0007730	1.0000000	0.0011280	0.9999989	0.1434573	0.0000044	NA	NA	0.0010251	0.9999997	0.0047736	1.0000000
blasted_case8	0.0012732	1.0000000	0.0013943	1.0000000	0.0493863	0.9999862	NA	NA	0.0012684	1.0000000	0.0110409	1.0000000
blasted_case9	0.0012663	1.0000000	0.0012100	1.0000000	0.0766977	0.0000000	NA	NA	0.0011743	1.0000000	NA	NA
blasted_squaring22	0.0016196	1.0000000	NA	NA	0.1697998	0.0000000	NA	NA	0.0013724	1.0000000	NA	NA
blasted_squaring26	0.0037136	1.0000000	0.8371974	0.0000000	0.5721998	0.0000000	NA	NA	0.0033812	1.0000000	NA	NA
blasted_squaring50	0.0027198	1.0000000	0.0690323	0.0000000	0.1661534	0.0000000	NA	NA	0.0026105	1.0000000	NA	NA
blasted_squaring51	0.0026033	1.0000000	0.1503676	0.0000000	0.3244400	0.0000000	NA	NA	0.0025400	1.0000000	NA	NA
busybox	0.0010854	1.0000000	0.0016437	1.0000000	1.0000000	0.0000000	NA	NA	0.0011415	1.0000000	NA	NA
DellSPLOT	0.0003014	0.9961108	0.0004283	0.9855897	0.2338054	0.8461812	NA	NA	0.0002471	0.9982106	0.0032351	0.9619242
embtoolkit-onlybool	0.0013525	1.0000000	NA	NA	0.9884235	1.0000000	NA	NA	0.5273388	0.0000000	NA	NA
fiasco	0.0008427	0.9999933	0.0006547	0.9999996	0.9867368	0.2969574	NA	NA	0.0009342	0.9999792	NA	NA
GuidanceService2.sk_2_27	0.0016772	1.0000000	0.0013097	1.0000000	0.5229459	0.0000000	NA	NA	0.0017600	1.0000000	NA	NA
jhipster	0.0005612	0.9997344	0.0009312	0.9930437	0.0430483	0.0000000	0.0055432	0.9998684	0.0007651	0.9979174	0.0078519	0.9985606
LargeAutomotive	0.0010105	1.0000000	NA	NA	1.0000000	0.0000000	NA	NA	0.0012346	1.0000000	NA	NA
polynomial.sk_7_25	0.0007879	0.9999938	0.0013146	0.9989725	0.1717230	0.9999104	NA	NA	0.0005736	0.9999998	NA	NA
registerlesSwap.sk_3_10	0.0013104	0.9999947	0.4208369	0.0000000	0.0812949	0.9999998	NA	NA	0.0010537	0.9999998	NA	NA
s1196a_3_2	0.0019010	1.0000000	0.0015319	1.0000000	0.0174924	0.0000000	NA	NA	0.0018479	1.0000000	NA	NA
s1196a_7_4	0.0018194	1.0000000	0.0017421	1.0000000	0.0170746	0.0044829	NA	NA	0.0018878	1.0000000	NA	NA
s1238a_3_2	0.0016355	1.0000000	0.3349503	0.0000000	0.0792596	0.0000000	NA	NA	0.0016896	1.0000000	NA	NA
s1238a_7_4	0.0019527	1.0000000	0.0970689	0.0000000	0.0302714	0.0000000	NA	NA	0.0022505	1.0000000	NA	NA
s1488_15_7	0.0039171	1.0000000	0.0024831	1.0000000	0.0046338	1.0000000	NA	NA	0.0029006	1.0000000	NA	NA
s1488_7_4	0.0032145	1.0000000	0.0022372	1.0000000	0.0074809	0.9997675	NA	NA	0.0020517	1.0000000	NA	NA
s27_15_7	0.0001402	0.9949001	0.0003224	0.9550624	0.0000000	1.0000000	0.0010081	0.9999820	0.0002767	0.9692023	0.0071316	0.7893139
s27_3_2	0.0000499	0.9997297	0.0004916	0.8809164	0.0000000	1.0000000	0.0010081	0.9999820	0.0005180	0.8667991	0.0028538	0.9737092
s27_7_4	0.0006485	0.7900245	0.0002599	0.9737350	0.0000000	1.0000000	0.0010081	0.9999820	0.0001219	0.9965380	0.0070115	0.7961063
s27_new_15_7	0.0005223	0.8644580	0.0002265	0.9815599	0.0000000	1.0000000	0.0025033	0.9999150	0.0003687	0.9379580	0.0060043	0.8715722
s27_new_3_2	0.0006655	0.7793464	0.0007565	0.7208783	0.0000000	1.0000000	0.0025033	0.9999150	0.0010005	0.5622203	0.0063797	0.8540243
s27_new_7_4	0.0005223	0.8644580	0.0003133	0.9580963	0.0000000	1.0000000	0.0025033	0.9999150	0.0001216	0.9965628	0.0102811	0.6451722
s298_15_7	0.0032591	1.0000000	0.0026927	1.0000000	0.0417725	0.0000000	NA	NA	0.0021713	1.0000000	0.0229020	1.0000000
s298_3_2	0.0012855	0.9999686	0.0005079	1.0000000	0.0387272	0.0000000	NA	NA	0.0013307	0.9999523	0.0092254	0.9999474
s298_7_4	0.0014466	0.9999999	0.0009939	1.0000000	0.0069437	0.0850060	NA	NA	0.0015550	0.9999995	0.0071903	1.0000000
s344_15_7	0.0029505	1.0000000	0.0021591	1.0000000	0.0492569	0.0000000	NA	NA	0.0015445	1.0000000	0.0175614	1.0000000
s344_3_2	0.0010539	1.0000000	0.0014445	1.0000000	0.0477145	0.0000000	NA	NA	0.0012916	1.0000000	0.0108842	1.0000000
s344_7_4	0.0011222	1.0000000	0.0016653	1.0000000	0.0620987	0.0000000	NA	NA	0.0011098	1.0000000	0.0144409	1.0000000
s349_15_7	0.0023320	1.0000000	0.0018605	1.0000000	0.0396747	0.0000000	NA	NA	0.0019910	1.0000000	0.0168187	1.0000000
s349_3_2	0.0014079	1.0000000	0.0008473	1.0000000	0.0317825	0.0000000	NA	NA	0.0013047	1.0000000	0.0090775	1.0000000
s349_7_4	0.0013145	1.0000000	0.0016555	1.0000000	0.0878584	0.0000000	NA	NA	0.0013237	1.0000000	0.0109518	1.0000000
s382_15_7	0.0022877	1.0000000	0.0029245	1.0000000	0.0925399	0.0000000	NA	NA	0.0019491	1.0000000	NA	NA
s382_3_2	0.0011023	0.9999974	0.0008544	0.9999999	0.0342812	0.0000000	NA	NA	0.0009291	0.9999997	0.0096036	0.9998449
s382_7_4	0.0009355	1.0000000	0.0011428	1.0000000	0.1626970	0.0000000	NA	NA	0.0010347	1.0000000	0.0054702	1.0000000
s420_15_7	0.0020879	1.0000000	0.0017349	1.0000000	0.8076502	0.0000000	NA	NA	0.0018350	1.0000000	NA	NA
s420_3_2	0.0012572	1.0000000	0.0012299	1.0000000	0.1325137	0.0000000	NA	NA	0.0007952	1.0000000	0.0080565	1.0000000
s420_7_4	0.0014360	1.0000000	0.0012755	1.0000000	0.5136831	0.0000000	NA	NA	0.0012335	1.0000000	NA	NA
s420_new_15_7	0.0018205	1.0000000	0.0017795	1.0000000	0.8642182	0.0000000	NA	NA	0.0016005	1.0000000	NA	NA
s420_new_3_2	0.0008616	1.0000000	0.0010368	1.0000000	0.3542817	0.0000000	NA	NA	0.0010032	1.0000000	0.0081586	1.0000000
s420_new_7_4	0.0013269	1.0000000	0.0016775	1.0000000	0.4351491	0.0000000	NA	NA	0.0011869	1.0000000	NA	NA
s420_new1_15_7	0.0021372	1.0000000	0.0018534	1.0000000	0.8187227	0.0000000	NA	NA	0.0018961	1.0000000	NA	NA
s420_new1_3_2	0.0007740	1.0000000	0.0010303	1.0000000	0.4596475	0.0000000	NA	NA	0.0013100	1.0000000	0.0083575	1.0000000
s420_new1_7_4	0.0014197	1.0000000	0.0013661	1.0000000	0.4152869	0.0000000	NA	NA	0.0012239	1.0000000	NA	NA
s444_15_7	0.0025486	1.0000000	0.0018473	1.0000000	0.0391142	0.0000000	NA	NA	0.0022436	1.0000000	NA	NA
s444_3_2	0.0010966	0.9999999	0.0010123	1.0000000	0.0291043	0.0000000	NA	NA	0.0010265	1.0000000	0.0058442	1.0000000
s444_7_4	0.0014838	1.0000000	0.0015919	1.0000000	0.0577748	0.0000000	NA	NA	0.0017753	0.9999999	NA	NA
s510_15_7	0.0014740	1.0000000	0.0012146	1.0000000	0.2817584	0.0000000	NA	NA	0.0008571	1.0000000	NA	NA
s510_3_2	0.0015700	0.9999976	0.0014125	0.9999996	0.5610223	0.0000000	NA	NA	0.0010583	1.0000000	0.0055833	1.0000000
s510_7_4	0.0011362	1.0000000	0.0014843	0.9999994	0.6334515	0.0000000	NA	NA	0.0007364	1.0000000	NA	NA
s526_15_7	0.0019668	1.0000000	0.1423881	0.0000000	0.3579287	0.0000000	NA	NA	0.0021603	1.0000000	NA	NA
s526_3_2	0.0011863	1.0000000	0.0014572	1.0000000	0.0700828	0.0000000	NA	NA	0.0013871	1.0000000	NA	NA
s526_7_4	0.0016381	1.0000000	0.0015038	1.0000000	0.0947843	0.0000000	NA	NA	0.0018834	1.0000000	NA	NA
s526a_15_7	0.0024347	1.0000000	0.4287505	0.0000000	0.3972689	0.0000000	NA	NA	0.0029217	1.0000000	NA	NA
s526a_3_2	0.0013885	1.0000000	0.0020875	0.9999993	0.0780826	0.0000000	NA	NA	0.0009820	1.0000000	NA	NA
s526a_7_4	0.0020097	1.0000000	0.0013201	1.0000000	0.1999664	0.0000000	NA	NA	0.0018324	1.0000000	NA	NA
s641_15_7	0.0016424	1.0000000	0.0016678	1.0000000	0.0761401	0.0000000	NA	NA	0.0018326	1.0000000	NA	NA
s641_3_2	0.0015115	1.0000000	0.0014608	1.0000000	0.2513842	0.0000000	NA	NA	0.0015287	1.0000000	NA	NA
s641_7_4	0.0015445	1.0000000	0.0012454	1.0000000	0.0695304	0.0000000	NA	NA	0.0014589	1.0000000	NA	NA
s713_15_7	0.0018409	1.0000000	0.0022917	1.0000000	0.3510273	0.0000000	NA	NA	0.0021685	1.0000000	NA	NA
s713_3_2	0.0013083	1.0000000	0.0013266	1.0000000	0.0461828	0.0000000	NA	NA	0.0014265	1.0000000	NA	NA
s713_7_4	0.0014334	1.0000000	0.0012237	1.0000000	0.0515399	0.0000000	NA	NA	0.0016370	1.0000000	NA	NA
s820a_15_7	0.0022197	1.0000000	0.0020287	1.0000000	0.3824322	0.0000000	NA	NA	0.0027158	1.0000000	NA	NA
s820a_3_2	0.0016802	0.9999739	0.0006194	1.0000000	0.2090161	0.0000000	NA	NA	0.0010361	1.0000000	NA	NA
s820a_7_4	0.0012469	1.0000000	0.0012230	1.0000000	0.2343982	0.0000000	NA	NA	0.0014683	0.9999999	NA	NA
s832a_15_7	0.0019201	1.0000000	0.0028947	1.0000000	0.4346639	0.0000000	NA	NA	0.0021888	1.0000000	NA	NA
s832a_3_2	0.0014557	1.0000000	0.0014439	1.0000000	0.0551578	0.0000000	NA	NA	0.0010952	1.0000000	NA	NA
s832a_7_4	0.0016064	1.0000000	0.0009699	1.0000000	0.4412826	0.0000000	NA	NA	0.0018659	1.0000000	NA	NA
s838_15_7	0.0024001	1.0000000	0.0021370	1.0000000	0.7384079	0.0000000	NA	NA	0.0024456	1.0000000	NA	NA
s838_3_2	0.0012048	1.0000000	0.0011324	1.0000000	0.3422012	0.0000000	NA	NA	0.0010543	1.0000000	NA	NA
s838_7_4	0.0011270	1.0000000	0.0009914	1.0000000	0.3631068	0.0000000	NA	NA	0.0011334	1.0000000	NA	NA
s953a_3_2	0.0011704	1.0000000	0.0013556	1.0000000	0.1260736	0.0000000	NA	NA	0.0007519	1.0000000	NA	NA
s953a_7_4	0.0014209	1.0000000	0.0011910	1.0000000	0.3234640	0.0000000	NA	NA	0.0015761	1.0000000	NA	NA
tableBasedAddition.sk_240_1024	0.0006929	1.0000000	0.0007742	1.0000000	0.2113487	0.0000000	NA	NA	0.0008411	1.0000000	NA	NA
toybox	0.0013390	1.0000000	0.0009941	1.0000000	0.2907412	0.0000000	NA	NA	0.0008472	1.0000000	NA	NA
uClibc	0.0011507	1.0000000	0.0008992	1.0000000	1.0000000	1.0000000	NA	NA	0.0010862	1.0000000	NA	NA

References

[1] T. M. Cover and J. A. Thomas, Elements of information theory. Wiley, 2006.

[2] I. Grosse, P. Bernaola-Galvan, P. Carpena, R. Roman-Roldan, J. Oliver, and H. E. Stanley, “Analysis of symbolic sequences using the jensen-shannon divergence,” Physical Review E, vol. 65, no. 2, pp. 041905/1–0419051/16, 2002.

[3] R. B. D’Agostino, Goodness-of-fit-techniques. CRC Press, 1986.

RQ3: Samplers’ uniformity

Ruben Heradio (rheradio@issi.uned.es)

Date: 2021-02-20